{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 词嵌入\n",
    "\n",
    "词嵌入在PyTorch中通过函数 `nn.Embdding(m, n)` 来实现的。`m`表示所有的单词数目，n表示词嵌入的维度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.8044  0.0947 -0.2797  0.4324 -0.5038\n",
      "[torch.FloatTensor of size 1x5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "word_to_ix = {'hello' : 0, 'world' : 1}\n",
    "embeds = nn.Embedding(2, 5)\n",
    "hello_idx = torch.LongTensor([word_to_ix['hello']])\n",
    "hello_idx = Variable(hello_idx)\n",
    "hello_embed = embeds(hello_idx)\n",
    "print(hello_embed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## N Gram 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONTEXT_SIZE = 2\n",
    "EMBEDDING_DIM = 10\n",
    "\n",
    "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
    "And dig deep trenches in thy beauty's field,\n",
    "Thy youth's proud livery so gazed on now,\n",
    "Will be a totter'd weed of small worth held:\n",
    "Then being asked, where all thy beauty lies,\n",
    "Where all the treasure of thy lusty days;\n",
    "To say, within thine own deep sunken eyes,\n",
    "Were an all-eating shame, and thriftless praise.\n",
    "How much more praise deserv'd thy beauty's use,\n",
    "If thou couldst answer 'This fair child of mine\n",
    "Shall sum my count, and make my old excuse,'\n",
    "Proving his beauty by succession thine!\n",
    "This were to be new made when thou art old,\n",
    "And see thy blood warm when thou feel'st it cold.\"\"\".split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram = [((test_sentence[i], test_sentence[i+1]), test_sentence[i+2])\n",
    "           for i in range(len(test_sentence) - 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocb = set(test_sentence)\n",
    "word_to_ix = {word : i for i, word in enumerate(vocb)}\n",
    "idx_to_word = {word_to_ix[word] : word for word in word_to_ix}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NgramModel(nn.Module) :\n",
    "    def __init__(self, vocb_size, context_size, n_dim) :\n",
    "        super().__init__()\n",
    "        self.n_word = vocb_size\n",
    "        self.embedding = nn.Embedding(self.n_word, n_dim)\n",
    "        self.linear1 = nn.Linear(context_size * n_dim, 128)\n",
    "        self.linear2 = nn.Linear(128, self.n_word)\n",
    "    \n",
    "    def forward(self, x) :\n",
    "        emb = self.embedding(x)\n",
    "        emb = emb.view(1, -1)\n",
    "        out = self.linear1(emb)\n",
    "        out = F.relu(out)\n",
    "        out = self.linear2(out)\n",
    "        log_prob = F.log_softmax(out, 1)\n",
    "        return log_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = NgramModel(len(vocb), CONTEXT_SIZE, EMBEDDING_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-2, weight_decay=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 20, Loss : 0.809687\n",
      "epoch: 40, Loss : 0.152495\n",
      "epoch: 60, Loss : 0.095547\n",
      "epoch: 80, Loss : 0.076622\n",
      "epoch: 100, Loss : 0.066594\n",
      "epoch: 120, Loss : 0.060094\n",
      "epoch: 140, Loss : 0.055386\n",
      "epoch: 160, Loss : 0.051820\n",
      "epoch: 180, Loss : 0.049030\n",
      "epoch: 200, Loss : 0.046760\n"
     ]
    }
   ],
   "source": [
    "epoches = 200\n",
    "for epoch in range(epoches) :\n",
    "    train_loss = 0\n",
    "    for word, label in trigram : \n",
    "        word = Variable(torch.LongTensor([word_to_ix[i] for i in word]))\n",
    "        label = Variable(torch.LongTensor([word_to_ix[label]]))\n",
    "        out = net(word)\n",
    "        loss = criterion(out, label)\n",
    "        train_loss += loss.data[0]\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    if (epoch + 1) % 20 == 0 :\n",
    "        print('epoch: {}, Loss : {:.6f}'.format(epoch + 1, train_loss / len(trigram)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = net.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input: ('so', 'gazed')\n",
      "input: on\n",
      "\n",
      "67\n",
      "real word is \"on\", predicted word is \"on\"\n"
     ]
    }
   ],
   "source": [
    "word, label = trigram[19]\n",
    "print('input: {}'.format(word))\n",
    "print('input: {}'.format(label), end =\"\\n\\n\")\n",
    "\n",
    "word = Variable(torch.LongTensor([word_to_ix[i] for i in word]))\n",
    "out = net(word)\n",
    "\n",
    "pred_label_idx = out.max(1)[1].data[0]\n",
    "print(pred_label_idx)\n",
    "\n",
    "predict_word = idx_to_word[pred_label_idx]\n",
    "print('real word is \"{}\", predicted word is \"{}\"'.format(label, predict_word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
